package com.spider.moudle.crawler.news;

import com.alibaba.fastjson.JSONObject;
import com.spider.moudle.api.Downloader;
import com.spider.moudle.api.realize.AbstractCrawler;
import com.spider.moudle.constant.Constant;
import com.spider.moudle.entity.*;
import com.spider.moudle.factory.DownloaderFactory;
import com.spider.moudle.factory.ExecutorServiceFactoy;
import com.spider.moudle.factory.NamedThreadFactory;
import com.spider.moudle.task.TaskManager;
import com.spider.moudle.util.CommonUtil;
import com.spider.moudle.util.DateUtil;
import com.spider.moudle.util.ErrorLogUtil;
import com.spider.moudle.util.SettingsUtil;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ExecutorService;

/**
 * 乌有之乡(编码utf-8)
 * Created by qiuhao on 2018/1/2.
 */
public class WuYouZXCrawler extends AbstractCrawler {

    /**
     * 这个是每个网站的个性化配置，是从settings.json里面读取到的(必须)
     */
    private Setting setting = SettingsUtil.getSettingBySite("WuYouZX");

    /**
     * 用户主页地址(非必须)
     */
    private String index = setting.getIndex();


    /**
     * 专题
     */
    private String topicIndex = "http://www.wyzxwk.com/s/index.html";

    @Override
    public void generateTask() {
        List<String> list=new ArrayList<>();
        ExecutorService executorService = null;
        try {
            ResponseEntity entity = null;
            try {
                entity = downloader.get(index);

            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }

            if (entity != null && entity.getStatus_code() == 200) {
                Document document = entity.getDocument();
                Elements elements = document.select(".g-sd-nav.s-shadow");
                if (elements.isEmpty()) {
                    return;
                }
                Elements liElements = elements.get(0).select("li");
                if (liElements != null && liElements.size() > 0) {
                    for (int i = 0; i < liElements.size(); i++) {
                        String url = liElements.get(i).select("a").get(0).absUrl("href");
                        list.add(url);

                    }
                }
                ResponseEntity entityy = null;
                try {
                    entityy = downloader.get(topicIndex);

                } catch (Exception e) {
                    log.error(e.getMessage(), e);
                }
                if (entityy != null && entityy.getStatus_code() == 200) {

                    Document doc = entityy.getDocument();
                    Elements topicelements = doc.select(".m-lstsp.g-mn.s-shadow");
                    if (topicelements.isEmpty()) {
                        return;
                    }
                    Elements spanElements = topicelements.get(0).select("span");
                    if (!spanElements.isEmpty()) {
                        for (int i = 0; i < spanElements.size(); i++) {
                            String url = spanElements.get(i).select("a").get(0).absUrl("href");
                            list.add(url);
                        }
                    }
                }
                executorService = ExecutorServiceFactoy.buildExecutorService(setting.getTask_thread_count(), ExecutorServiceFactoy.getName(setting.getName(), setting.getIndex()),
                        new NamedThreadFactory(setting.getName(), "生成任务"));

                for(int i=0;i<list.size();i++){
                    WuYouZXTaskCrawler default_task = new WuYouZXTaskCrawler(list.get(i));
                    executorService.submit(default_task);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (null != executorService) {
                executorService.shutdown();
                while (!executorService.isTerminated()) {
                    try {
                        Thread.sleep(1500);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    }

    @Override
    public JSONObject crawlItem(Task task) throws Exception {
        if (task.getTaskType().equals(TaskType.SPEECH)) {

            JSONObject item = new JSONObject();
            SpeechBean speechBean = new SpeechBean();
            String url = task.getUrl();

            speechBean.setUrl(url);
            speechBean.setId(task.get("id").toString());
            speechBean.setRead_count(Integer.valueOf(task.get("read_count").toString()));
            ResponseEntity entity = null;
            try {
                entity = downloader.get(url);
            } catch (Exception e) {
                System.out.println(url);
                ErrorLogUtil.error(e.getMessage(), e);
            }

            if (entity != null && entity.getStatus_code() == 200) {
                try {
                    Document document = entity.getDocument();

                    String title = CommonUtil.trim(document.select("h1").get(0).text());
                    String publish_time = "";
                    String publish_user = "";
                    String content = "";
                    int reproduce_count=0;
                    List<String> img_urls = new ArrayList<>();
                    Elements greyElements = document.select(".s-grey-3");
                    if (greyElements != null && greyElements.size() > 0) {
                        publish_time = greyElements.get(2).text() + " 00:00:00";
                    }

                    Elements fcbElements = document.select(".m-atc.f-cb");
                    Elements propertyElements = document.select(".property");
                    if (!fcbElements.isEmpty()) {
                        fcbElements.get(0).select(".f-fl").select("span").remove();
                        publish_user = CommonUtil.trim(fcbElements.get(0).select(".f-fl").text());
                    }else if(!propertyElements.isEmpty()){
                        Elements span=propertyElements.get(0).select("span");
                        publish_time=span.get(1).text().split("：")[1]+" 00:00:00";
                        publish_user=span.get(2).text().split("：")[1];
                    }

                    Elements reproduceElements= document.select(".jiathis_bubble_style");
                    if(!reproduceElements.isEmpty()){
                        reproduce_count=Integer.valueOf(CommonUtil.trim(reproduceElements.get(0).text()));
                    }

                    Elements aritcleElements = document.select("article");
                    Elements contentElements = document.select(".text_content");
                    if (!aritcleElements.isEmpty()) {
                        aritcleElements.get(0).select(".m-weixincode").remove();
                        aritcleElements.get(0).select(".cyReward").remove();
                        aritcleElements.get(0).select("#cyReward").remove();
                        aritcleElements.get(0).select(".s-grey-2").remove();
                        aritcleElements.get(0).select(".s-grey-2.f-fr").remove();
                        aritcleElements.get(0).select(".m-pages-a").remove();
                        aritcleElements.get(0).select("#labsTarget").remove();
                        aritcleElements.get(0).select(".m-rlt-2").remove();

                        content = aritcleElements.get(0).text();

                        Elements imgElements = aritcleElements.get(0).select("img");
                        if(!imgElements.isEmpty()){
                            for(int i=0;i<imgElements.size();i++){
                                String img=imgElements.get(i).attr("src");
                                img_urls.add(img);
                            }
                        }

                    }else if(!contentElements.isEmpty()){
                        contentElements.get(0).select(".statement").remove();
                        contentElements.get(0).select("#bottom_tool").remove();
                        Elements baseLine=contentElements.get(0).select("[align=baseline]");
                        if(CommonUtil.isNotNull(baseLine)){
                            baseLine.remove();
                        }

                        content= contentElements.get(0).text();
                        Elements imgElements = contentElements.get(0).select("img");
                        if(!imgElements.isEmpty()){
                            for(int i=0;i<imgElements.size();i++){
                                String img=imgElements.get(i).attr("src");
                                img_urls.add(img);
                            }
                        }
                    }

                    speechBean.setPublish_user(publish_user);
                    speechBean.setTitle(CommonUtil.handle(title));
                    speechBean.setUrl(url);
                    try {
                        if(publish_time.equals("")){
                           return null;
                        }
                        speechBean.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss,
                                publish_time));
                    } catch (ParseException e) {
                        System.out.println("获取发布时间出错："+url);
                    }

                    Date now = new Date();

                    speechBean.setSpider_time(now);
                    speechBean.setReproduce_count(reproduce_count);
                    speechBean.setContent(CommonUtil.handle(content));
                    speechBean.setImg_urls(img_urls);

                    item.put("data", speechBean.toJSONObject());
                    return item;
                } catch (Exception e) {
                    e.printStackTrace();
                }

            }
        }

        return null;
    }


    /**
     * 生成任务
     */
    public class WuYouZXTaskCrawler implements Runnable {

        private String url;


        private Downloader downloader = DownloaderFactory.buildDownloader();
        int page = 1;

        public WuYouZXTaskCrawler(String url) {
            this.url = url;

        }

        @Override
        public void run() {

//            handle();
            history();

        }

        /**
         * 每天爬取的最新数据
         */
        private void handle() {
            String lastUrl = url;
            for (int p = 0; p < 2; p++) {
                boolean flag = false;
                try {
                    ResponseEntity entity = null;
                    try {
                        entity = downloader.get(lastUrl);
                    } catch (Exception e) {
                        log.error(e.getMessage(), e);
                    }
                    if (entity != null && entity.getStatus_code() == 200) {
                        Document document = entity.getDocument();
                        Elements ptElements = document.select(".m-pt");
                        if (ptElements != null && ptElements.size() > 0) {
                            for (int j = 0; j < ptElements.size(); j++) {
                                String news_url = ptElements.get(j).select("h3").get(0).select("a").get(0).absUrl("href");

                                if (CommonUtil.isNotNull(news_url)) {
                                    String[] params = news_url.split("/");
                                    String speechkey = setting.getSite() + "_" + params[params.length - 1].split("\\.")[0];
                                    Task task = new Task(news_url, TaskType.SPEECH, WebType.NEWS, speechkey);
                                    String id = params[params.length - 1].split("\\.")[0];
                                    task.put("id", id);
                                    String hudong = ptElements.get(j).select("span.s-grey-2").get(1).text();
                                    int read_count = 0;
                                    if (!hudong.contains("互动：")) {
                                        read_count = Integer.valueOf(ptElements.get(j).select("span.s-grey-2").get(2).text().split("互动：")[1]);
                                    } else {
                                        read_count = Integer.valueOf(ptElements.get(j).select("span.s-grey-2").get(1).text().split("互动：")[1]);
                                    }
                                    task.put("read_count", read_count);
                                    Elements em = ptElements.get(j).select("span.s-grey-2").get(0).select("em");
                                    if (em != null && em.size() > 0) {
                                        TaskManager.add(setting.getSpeech_task(), task);
                                    } else {
                                        String updateTimeString = ptElements.get(j).select("span.s-grey-2").get(0).text();
                                        if (CommonUtil.isNotNull(updateTimeString)) {
                                            Date updateTime = DateUtil.str_to_date(DateUtil.yyyy_MM_dd, updateTimeString);
                                            TaskManager.add(setting.getSpeech_task(), task);
                                            if (j == ptElements.size() - 1) {
                                                Date now = new Date();
                                                if (updateTime.getTime() < now.getTime() - Constant.LAST_48_HOUR_TIME_STAMP) {
                                                    break;
                                                }
                                                if (document.select(".u-next").size() > 1) {
                                                    String nextUrl = document.select(".u-next").get(1).absUrl("href");
                                                    lastUrl = nextUrl;
                                                    flag = true;
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
                if (!flag) {
                    break;
                }

            }


        }


        /**
         * 历史数据
         */
        private void history() {
            String lastUrl = url;
            try {
                ResponseEntity entity = null;
                try {
                    entity = downloader.get(lastUrl);
                } catch (Exception e) {
                    log.error(e.getMessage(), e);
                }
                if (entity != null && entity.getStatus_code() == 200) {
                    Document document = entity.getDocument();
                    Elements pgElements = document.select("div.m-pages");
                    if (pgElements.size() > 0) {
                        Elements aElements = pgElements.get(0).select("a");
                        if (aElements != null && aElements.size() > 0) {
                            page = Integer.valueOf(aElements.get(aElements.size() - 2).text());
                        }
                    }
                    for (int p = 1; p <= page; p++) {
                        ResponseEntity entityy = null;
                        try {
                            entityy = downloader.get(lastUrl, true);
                        } catch (Exception e) {
                            log.error(e.getMessage(), e);
                        }
                        if (entityy != null && entityy.getStatus_code() == 200) {
                            Document doc = entityy.getDocument();
                            Elements ptElements = doc.select(".m-pt");
                            if (ptElements != null && ptElements.size() > 0) {
                                for (int j = 0; j < ptElements.size(); j++) {
                                    String news_url = ptElements.get(j).select("h3").get(0).select("a").get(0).absUrl("href");
                                    String[] params = news_url.split("/");
                                    String speechkey = setting.getSite() + "_" + params[params.length - 1].split("\\.")[0];
                                    String id=params[params.length - 1].split("\\.")[0];
                                    Task task = new Task(news_url, TaskType.SPEECH, WebType.NEWS, speechkey);
                                    task.put("id",id);
                                    String hudong=ptElements.get(j).select("span.s-grey-2").get(1).text();
                                    int read_count= 0;
                                    if(!hudong.contains("互动：")){
                                        read_count = Integer.valueOf(ptElements.get(j).select("span.s-grey-2").get(2).text().split("互动：")[1]);
                                    }else{
                                        read_count = Integer.valueOf(ptElements.get(j).select("span.s-grey-2").get(1).text().split("互动：")[1]);
                                    }

                                    task.put("read_count",read_count);
                                    TaskManager.add(setting.getSpeech_task(), task);
                                }

                                if(url.contains("tagname")){
                                    lastUrl = url + "&page="+p;
                                }else {
                                    lastUrl = url + "index_" + p + 1 + ".html";
                                }
                            }
                        }
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}
